Convolutional Neural Networks¶

import numpy as np

from glob import glob

​

# load filenames for human and dog images

human_files = np.array(glob("lfw/*/*"))

dog_files = np.array(glob("dogImages/*/*/*"))

​

# print number of images in each dataset

print('There are %d total human images.' % len(human_files))

print('There are %d total dog images.' % len(dog_files))

import cv2                

import matplotlib.pyplot as plt                        

%matplotlib inline                               

​

# extract pre-trained face detector

face_cascade = cv2.CascadeClassifier('haarcascades/haarcascade_frontalface_alt.xml')

​

# load color (BGR) image

img = cv2.imread(human_files[10])

# convert BGR image to grayscale

gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)

​

# find faces in image

faces = face_cascade.detectMultiScale(gray)

​

# print number of faces detected in the image

print('Number of faces detected:', len(faces))

​

# get bounding box for each detected face

for (x,y,w,h) in faces:

    # add bounding box to color image

    cv2.rectangle(img,(x,y),(x+w,y+h),(255,0,0),2)

# convert BGR image to RGB for plotting

cv_rgb = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)

​

# display the image, along with bounding box

plt.imshow(cv_rgb)

plt.show()

# returns "True" if face is detected in image stored at img_path

def face_detector(img_path):

    img = cv2.imread(img_path)

    gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)

    faces = face_cascade.detectMultiScale(gray)

    return len(faces) > 0

from tqdm import tqdm

​

human_files_short = human_files[:100]

dog_files_short = dog_files[:100]

​

#-#-# Do NOT modify the code above this line. #-#-#

​

## TODO: Test the performance of the face_detector algorithm 

## on the images in human_files_short and dog_files_short.

​

def visualize_img(img_path, countX = 4):

    fig = plt.figure()

    n_images = len(img_path)

    for i in range(n_images):

        ax = fig.add_subplot(np.ceil(n_images/float(countX)), countX, i + 1)

        img = cv2.imread(img_path[i])

        plt.imshow(cv2.cvtColor(img, cv2.COLOR_BGR2RGB))

        plt.axis("off")

    fig.set_size_inches(np.array(fig.get_size_inches()) * n_images)

#     fig.show()

def check_accurary(img_path, detector = face_detector, kind = 'dog'):

    true_detected = np.array([detector(path) for path in img_path])

    if (kind == 'human'):

        img = [path for i,path in enumerate(img_path, 0) if true_detected[i] == 0 ]

    else: img = [path for i,path in enumerate(img_path, 0) if true_detected[i] != 0 ]

    fig = plt.figure()

    n_images = len(img)

    visualize_img(img)

    return true_detected.sum()/len(img_path)

​

print("Human face detects : {} %".format(check_accurary(human_files_short, face_detector, 'human')*100))

print("Dog detects as human: {} %".format(check_accurary(dog_files_short, face_detector, 'dog')*100))

​

### (Optional) 

### TODO: Test performance of another face detection algorithm.

### Feel free to use as many code cells as needed.

import torch

import torchvision.models as models

​

# define VGG16 model

VGG16 = models.vgg16(pretrained=True)

​

# check if CUDA is available

use_cuda = torch.cuda.is_available()

​

# move model to GPU if CUDA is available

if use_cuda:

    VGG16 = VGG16.cuda()

from PIL import Image

import torchvision.transforms as transforms

​

# Set PIL to be tolerant of image files that are truncated.

from PIL import ImageFile

ImageFile.LOAD_TRUNCATED_IMAGES = True

​

def VGG16_predict(img_path):

    '''

    Use pre-trained VGG-16 model to obtain index corresponding to 

    predicted ImageNet class for image at specified path

    Args:

        img_path: path to an image

    Returns:

        Index corresponding to VGG-16 model's prediction

    '''

    ## TODO: Complete the function.

    ## Load and pre-process an image from the given img_path

    ## Return the *index* of the predicted class for that image

    # mini-batches of 3-channel RGB images of shape (3 x H x W), 

    # where H and W are expected to be at least 224. The images 

    # have to be loaded in to a range of [0, 1] and then normalized 

    # using mean = [0.485, 0.456, 0.406] and std = [0.229, 0.224, 0.225]

    normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406],

                                     std=[0.229, 0.224, 0.225])

    tf = transforms.Compose([transforms.Resize(224), transforms.CenterCrop(224), transforms.ToTensor(), normalize])   

    img = tf(Image.open(img_path)).unsqueeze(0)

    if use_cuda:

        img = img.cuda()

    predictor = VGG16(img)

    return predictor.argmax() # predicted class index

### returns "True" if a dog is detected in the image stored at img_path

def dog_detector(img_path):

    ## TODO: Complete the function.

    rs = VGG16_predict(img_path).cpu().numpy() #convert cuda instance to cpu -> numpy

    return (rs >= 151 and rs <= 268)

### TODO: Test the performance of the dog_detector function

### on the images in human_files_short and dog_files_short.

​

dog = []

human = []

for dog_path, human_path in zip(dog_files_short, human_files_short):

    dog.append(dog_detector(dog_path)) 

    human.append(dog_detector(human_path)) 

​

img_hu = [path for i,path in enumerate(human_files_short, 0) if human[i]]

print("Human detects as dog {} %".format(len(img_hu)/len(human_files_short)*100))

print("Missed classified human as dog (if any):")

if (len(img_hu) > 0): visualize_img(img_hu)

​

img_dog = [path for i,path in enumerate(dog_files_short, 0) if ~dog[i]]    

print("Dog detects as dog {} %".format(100-len(img_dog)/len(dog_files_short)*100))

print("Missed classified dog (if any):")

if (len(dog) > 0): visualize_img(img_dog)

### (Optional) 

### TODO: Report the performance of another pre-trained network.

### Feel free to use as many code cells as needed.

​

#define model

inception = models.inception_v3(pretrained=True)

​

#check cuda

if use_cuda:

    inception = inception.cuda()

    inception.eval()

def inception_predict(img_path):

    '''

    Use pre-trained Inception model to obtain index corresponding to 

    predicted ImageNet class for image at specified path

    Args:

        img_path: path to an image

    Returns:

        Index corresponding to VGG-16 model's prediction

    '''

    ## TODO: Complete the function.

    ## Load and pre-process an image from the given img_path

    ## Return the *index* of the predicted class for that image

    # mini-batches of 3-channel RGB images of shape (3 x H x W), 

    # where H and W are expected to be at least 299. The images 

    # have to be loaded in to a range of [0, 1] and then normalized 

    # using mean = [0.485, 0.456, 0.406] and std = [0.229, 0.224, 0.225]

    normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406],

                                     std=[0.229, 0.224, 0.225])

    tf = transforms.Compose([transforms.Resize(299), transforms.CenterCrop(299), transforms.ToTensor(), normalize])   

    img = tf(Image.open(img_path).convert('RGB')).unsqueeze(0)

    if use_cuda:

        img = img.cuda()

    with torch.no_grad():

        predictor = inception(img)

    return predictor.argmax() # predicted class index

### returns "True" if a dog is detected in the image stored at img_path

def dog_detector_inception(img_path):

    ## TODO: Complete the function.

    rs = inception_predict(img_path).cpu().numpy() #convert cuda instance to cpu -> numpy

    return (rs >= 151 and rs <= 268)

### TODO: Test the performance of the dog_detector function

### on the images in human_files_short and dog_files_short.

​

dog = []

human = []

for dog_path, human_path in zip(dog_files_short, human_files_short):

    dog.append(dog_detector_inception(dog_path)) 

    human.append(dog_detector_inception(human_path)) 

​

img_hu = [path for i,path in enumerate(human_files_short, 0) if human[i]]

print("Human detects as dog {} %".format(len(img_hu)/len(human_files_short)*100))

print("Missed classified human as dog (if any):")

if (len(img_hu) > 0): visualize_img(img_hu)

​

img_dog = [path for i,path in enumerate(dog_files_short, 0) if ~dog[i]]    

print("Dog detects as dog {} %".format(100-len(img_dog)/len(dog_files_short)*100))

print("Missed classified dog (if any):")

if (len(dog) > 0): visualize_img(img_dog)

### (Optional) 

### TODO: Report the performance of another pre-trained network.

### Feel free to use as many code cells as needed.

​

#define model

resnet = models.wide_resnet50_2(pretrained=True)

​

#check cuda

if use_cuda:

    resnet = resnet.cuda()

    resnet.eval()

def resnet_predict(img_path):

    '''

    Use pre-trained Inception model to obtain index corresponding to 

    predicted ImageNet class for image at specified path

    Args:

        img_path: path to an image

    Returns:

        Index corresponding to VGG-16 model's prediction

    '''

    ## TODO: Complete the function.

    ## Load and pre-process an image from the given img_path

    ## Return the *index* of the predicted class for that image

    # mini-batches of 3-channel RGB images of shape (3 x H x W), 

    # where H and W are expected to be at least 224. The images 

    # have to be loaded in to a range of [0, 1] and then normalized 

    # using mean = [0.485, 0.456, 0.406] and std = [0.229, 0.224, 0.225]

    normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406],

                                     std=[0.229, 0.224, 0.225])

    tf = transforms.Compose([transforms.Resize(224), transforms.CenterCrop(224), transforms.ToTensor(), normalize])   

    img = tf(Image.open(img_path).convert('RGB')).unsqueeze(0)

    if use_cuda:

        img = img.cuda()

    with torch.no_grad():

        predictor = resnet(img)

    return predictor.argmax() # predicted class index

### returns "True" if a dog is detected in the image stored at img_path

def dog_detector_resnet(img_path):

    ## TODO: Complete the function.

    rs = resnet_predict(img_path).cpu().numpy() #convert cuda instance to cpu -> numpy

    return (rs >= 151 and rs <= 268)

### TODO: Test the performance of the dog_detector function

### on the images in human_files_short and dog_files_short.

​

dog = []

human = []

for dog_path, human_path in zip(dog_files_short, human_files_short):

    dog.append(dog_detector_resnet(dog_path)) 

    human.append(dog_detector_resnet(human_path)) 

​

img_hu = [path for i,path in enumerate(human_files_short, 0) if human[i]]

print("Human detects as dog {} %".format(len(img_hu)/len(human_files_short)*100))

print("Missed classified human as dog (if any):")

if (len(img_hu) > 0): visualize_img(img_hu)

img_dog = [path for i,path in enumerate(dog_files_short, 0) if ~dog[i]]    

print("Dog detects as dog {} %".format(100-len(img_dog)/len(dog_files_short)*100))

print("Missed classified dog (if any):")

if (len(dog) > 0): visualize_img(img_dog)

import os

from torchvision import datasets

from PIL import ImageFile

from glob import glob

​

ImageFile.LOAD_TRUNCATED_IMAGES = True

​

### TODO: Write data loaders for training, validation, and test sets

## Specify appropriate transforms, and batch_sizes

​

min_resize = 224

min_crop = 224

normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406],

                                std=[0.229, 0.224, 0.225])

​

input_dir = 'dogImages'

train_dir = 'train'

test_dir = 'test'

valid_dir = 'valid'

​

#transform

train_tf = transforms.Compose([transforms.Resize(min_resize),

                                       transforms.CenterCrop(min_crop),

                                       transforms.RandomHorizontalFlip(),

                                       transforms.RandomVerticalFlip(),

                                       transforms.RandomRotation(2020),

                                       transforms.ToTensor(),

                                       normalize])

​

test_tf = transforms.Compose([transforms.Resize(min_resize),

                                       transforms.CenterCrop(min_crop),

                                       transforms.ToTensor(),

                                       normalize])

​

valid_tf = test_tf = transforms.Compose([transforms.Resize(min_resize),

                                       transforms.CenterCrop(min_crop),

                                       transforms.ToTensor(),

                                       normalize])

​

#load datasets

train_dt = datasets.ImageFolder(input_dir + '/' + train_dir, transform=train_tf)

test_dt = datasets.ImageFolder(input_dir + '/' + test_dir, transform=test_tf)

valid_dt = datasets.ImageFolder(input_dir + '/' + valid_dir, transform=valid_tf)

​

print("Train data: {} images loaded".format(len(train_dt)))

print("Test data: {} images loaded".format(len(test_dt)))

print("Valid data: {} images loaded".format(len(valid_dt)))

​

loaders_scratch = {}

loaders_scratch[train_dir] = torch.utils.data.DataLoader(train_dt, batch_size=32, shuffle=True)

loaders_scratch[test_dir] = torch.utils.data.DataLoader(test_dt, batch_size=32)

loaders_scratch[valid_dir] = torch.utils.data.DataLoader(valid_dt, batch_size=32)

​

print("Total categories: {}".format(len(train_dt.classes)))

print(train_dt.classes)

import torch.nn as nn

import torch.nn.functional as F

​

# define the CNN architecture

class Net(nn.Module):

    ### TODO: choose an architecture, and complete the class

    def __init__(self):

        super(Net, self).__init__()

        ## Define layers of a CNN

        #Input channels = 3, output channels = 16 (224x224x3)

        self.conv1 = torch.nn.Conv2d(3, 16, kernel_size=3, stride=1, padding=1)

        #Input channels = 16, output channels = 32 (112x112x16)

        self.conv2 = torch.nn.Conv2d(16, 32, kernel_size=3, stride=1, padding=1)

        #Input channels = 32, output channels = 64 (56x56x32)

        self.conv3 = torch.nn.Conv2d(32, 64, kernel_size=3, stride=1, padding=1)

        #Input channels = 64, output channels = 128 (28x28x64) => (14x14x128)

        self.conv4 = torch.nn.Conv2d(64, 128, kernel_size=3, stride=1, padding=1)        

        #Max pooling 

        self.pool = nn.MaxPool2d(2, 2)

        #Linear layer

        self.fc1 = torch.nn.Linear(14 * 14 * 128, 1000)

        self.fc2 = torch.nn.Linear(1000, 133)

        #Dropout

        self.dropout = nn.Dropout(p = 0.4)

        #Batch norm 

        self.batch_norm = nn.BatchNorm1d(num_features=1000)

    def forward(self, x):

        ## Define forward behavior

        x = F.relu(self.conv1(x))

        x = self.pool(x)

        x = F.relu(self.conv2(x))

        x = self.pool(x)

        x = F.relu(self.conv3(x))

        x = self.pool(x)

        x = F.relu(self.conv4(x))

        x = self.pool(x)

        #Reshape data to input to the input layer of the neural net

        #Size changes from (14, 14, 128) to (1, 25088)

        #Recall that the -1 infers this dimension from the other given dimension

        x = x.view(-1, 14 * 14 *128)

        #Dropout layer

        x = self.dropout(x)

        #Hidden layer

        x = F.relu(self.batch_norm(self.fc1(x)))

        #Dropout layer

        x = self.dropout(x)

        #Hidden layer

        x = self.fc2(x)

        return x

​

#-#-# You do NOT have to modify the code below this line. #-#-#

​

# instantiate the CNN

model_scratch = Net()

​

# move tensors to GPU if CUDA is available

if use_cuda:

    model_scratch.cuda()

print(model_scratch)

import torch.optim as optim

​

learning_rate = 0.01

### TODO: select loss function

criterion_scratch = nn.CrossEntropyLoss()

​

### TODO: select optimizer

optimizer_scratch = optim.SGD(model_scratch.parameters(), lr=learning_rate)

# the following import is required for training to be robust to truncated images

from PIL import ImageFile

ImageFile.LOAD_TRUNCATED_IMAGES = True

​

def train(n_epochs, loaders, model, optimizer, criterion, use_cuda, save_path):

    """returns trained model"""

    # initialize tracker for minimum validation loss

    valid_loss_min = np.Inf 

    for epoch in range(1, n_epochs+1):

        # initialize variables to monitor training and validation loss

        train_loss = 0.0

        valid_loss = 0.0

        ###################

        # train the model #

        ###################

        model.train()

        for batch_idx, (data, target) in enumerate(loaders['train']):

            # move to GPU

            if use_cuda:

                data, target = data.cuda(), target.cuda()

            ## find the loss and update the model parameters accordingly

            ## record the average training loss, using something like

            ## train_loss = train_loss + ((1 / (batch_idx + 1)) * (loss.data - train_loss))

            #Set the parameter gradients to zero

            optimizer.zero_grad()

            #Forward pass, backward pass, optimize

            output = model(data)

            loss = criterion(output, target)

            loss.backward()

            optimizer.step()

            train_loss = train_loss + ((1 / (batch_idx + 1)) * (loss.data - train_loss))

        ######################    

        # validate the model #

        ######################

        model.eval()

        for batch_idx, (data, target) in enumerate(loaders['valid']):

            # move to GPU

            if use_cuda:

                data, target = data.cuda(), target.cuda()

            ## update the average validation loss

            val_output = model(data)

            val_loss = criterion(val_output, target)

            valid_loss = valid_loss + ((1 / (batch_idx + 1)) * (val_loss.data - valid_loss))

        # print training/validation statistics 

        print('Epoch: {} \tTraining Loss: {:.6f} \tValidation Loss: {:.6f}'.format(

            epoch, 

            train_loss,

            valid_loss

            ))

        ## TODO: save the model if validation loss has decreased

        if valid_loss <= valid_loss_min:

            print('Saving  ...')

            torch.save(model.state_dict(), save_path)

            valid_loss_min = valid_loss

        else:

            print()

    # return trained model

    return model

​

​

# train the model

model_scratch = train(50, loaders_scratch, model_scratch, optimizer_scratch, 

                      criterion_scratch, use_cuda, 'model_scratch.pt')

​

# load the model that got the best validation accuracy

model_scratch.load_state_dict(torch.load('model_scratch.pt'))

model_scratch.load_state_dict(torch.load('model_scratch.pt'))

def test(loaders, model, criterion, use_cuda):

​

    # monitor test loss and accuracy

    test_loss = 0.

    correct = 0.

    total = 0.

​

    model.eval()

    for batch_idx, (data, target) in enumerate(loaders['test']):

        # move to GPU

        if use_cuda:

            data, target = data.cuda(), target.cuda()

        # forward pass: compute predicted outputs by passing inputs to the model

        output = model(data)

        # calculate the loss

        loss = criterion(output, target)

        # update average test loss 

        test_loss = test_loss + ((1 / (batch_idx + 1)) * (loss.data - test_loss))

        # convert output probabilities to predicted class

        pred = output.data.max(1, keepdim=True)[1]

        # compare predictions to true label

        correct += np.sum(np.squeeze(pred.eq(target.data.view_as(pred))).cpu().numpy())

        total += data.size(0)

    print('Test Loss: {:.6f}\n'.format(test_loss))

​

    print('\nTest Accuracy: %2d%% (%2d/%2d)' % (

        100. * correct / total, correct, total))

​

# call test function    

test(loaders_scratch, model_scratch, criterion_scratch, use_cuda)

## TODO: Specify data loaders

loaders_transfer = loaders_scratch

import torchvision.models as models

import torch.nn as nn

​

## TODO: Specify model architecture 

model_transfer = models.wide_resnet50_2(pretrained=True)

print(model_transfer)

print(model_transfer.fc)

for param in model_transfer.parameters():

    param.requires_grad = False

​

model_transfer.fc = torch.nn.Linear(2048, 133, bias = True)

​

if use_cuda:

    model_transfer = model_transfer.cuda()

learning_rate = 0.01

criterion_transfer = nn.CrossEntropyLoss()

optimizer_transfer = optim.SGD(model_transfer.parameters(), lr=learning_rate)

# train the model

n_epochs = 50

model_transfer = train(n_epochs, loaders_transfer, model_transfer, optimizer_transfer, criterion_transfer, use_cuda, 'model_transfer.pt')

​

# load the model that got the best validation accuracy (uncomment the line below)

model_transfer.load_state_dict(torch.load('model_transfer.pt'))

test(loaders_transfer, model_transfer, criterion_transfer, use_cuda)

### TODO: Write a function that takes a path to an image as input

### and returns the dog breed that is predicted by the model.

​

# list of class names by index, i.e. a name can be accessed like class_names[0]

class_names = [item[4:].replace("_", " ") for item in train_dt.classes]

​

def predict_breed_transfer(img_path):

    # load the image and return the predicted breed

    normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406],

                                     std=[0.229, 0.224, 0.225])

    tf = transforms.Compose([transforms.Resize(224), transforms.CenterCrop(224), transforms.ToTensor(), normalize])   

    img = tf(Image.open(img_path)).unsqueeze(0)

    if use_cuda:

        img = img.cuda()

    model_transfer.eval()

    output = model_transfer(img)

    # convert output probabilities to predicted class

    pred = output.data.max(1, keepdim=True)[1]

    # compare predictions to true label

    return class_names[np.squeeze(pred.cpu().numpy())]

### TODO: Write your algorithm.

### Feel free to use as many code cells as needed.

​

dog_string = 'I guess you are a \n'

human_string = 'You look like a ... \n'

cant_detect_string = "Sorry I don't know who you are or what it is \nCome back next time when the system is upgraded"

​

​

def run_app(img_path):

    ## handle cases for a human face, dog, and neither

    if (face_detector(img_path)):

        title = 'Hello, human!'

        pred = predict_breed_transfer(img_path)

        output_string = human_string + pred

    elif (dog_detector(img_path)):

        title = 'Hi, there'

        pred = predict_breed_transfer(img_path)

        output_string = dog_string + pred

    else:

        title = 'Hi'

        output_string = cant_detect_string 

​

    plt.title(title + '\n' + output_string)

    plt.imshow(Image.open(img_path))

    plt.axis('off')

    plt.show()

## TODO: Execute your algorithm from Step 6 on

## at least 6 images on your computer.

## Feel free to use as many code cells as needed.

​

## suggested code, below

for file in np.hstack((human_files[10:20], dog_files[20:30])):

    run_app(file)

new_files = np.array(glob("images/*"))

for file in new_files[:4]:

    run_app(file)